notebook.community

Edit and run

Cohen's Kappa Statistics

Agreement Rate Calculation

Note: : this notebook assumes the use of Python 3

Preamble: Settings Django Environment





In [2]:



    


%load preamble_directives.py

Weighted Cohen's Kappa Function

$kappa = 1 - \frac{\sum W*X}{\sum W*M}$ where $*$ indicates the element-wise matrix multiplication.

$X$: Is the matrix of Observed Scores

$M$: Is the matrix of Score Agreement by Chance

$W$: Is the Weight Matrix.





In [3]:



    


from django.contrib.auth.models import User
from source_code_analysis.models import SoftwareProject



    











In [4]:



    


from evaluations import Judge

Calculate Agreement Score (Function Definition)





In [5]:



    


from evaluations import calculate_agreement_scores

Cohens' Kappa Function (definition)





In [6]:



    


from evaluations import cohens_kappa

Cohens' Kappa 3 WITHOUT logs





In [8]:



    


j1 = Judge('leonardo.nole', 'CoffeeMaker')
j2 = Judge('rossella.linsalata', 'CoffeeMaker')
J = calculate_agreement_scores(j1, j2)
unweighted_k = cohens_kappa(J)
weighted_k = cohens_kappa(J, weighted=True)
print('CoffeeMaker & %.3f & %.3f' % (unweighted_k, weighted_k))

j1 = Judge('leonardo.nole', 'JFreechart', '0.6.0')
j2 = Judge('antonio.petrone', 'JFreechart', '0.6.0')
J = calculate_agreement_scores(j1, j2)
unweighted_k = cohens_kappa(J)
weighted_k = cohens_kappa(J, weighted=True)
print('JFreechart (0.6.0) & %.3f & %.3f' % (unweighted_k, weighted_k))

j1 = Judge('leonardo.nole', 'JFreechart', '0.7.1')
j2 = Judge('antonio.petrone', 'JFreechart', '0.7.1')
J = calculate_agreement_scores(j1, j2)
unweighted_k = cohens_kappa(J)
weighted_k = cohens_kappa(J, weighted=True)
print('JFreechart (0.7.1) & %.3f & %.3f' % (unweighted_k, weighted_k))

j1 = Judge('leonardo.nole', 'JHotDraw', '7.4.1')
j2 = Judge('rossella.linsalata', 'JHotDraw', '7.4.1')
J = calculate_agreement_scores(j1, j2)
unweighted_k = cohens_kappa(J)
weighted_k = cohens_kappa(J, weighted=True)
print('JHotDraw (7.4.1) & %.3f & %.3f' % (unweighted_k, weighted_k))



    


















    






CoffeeMaker & 1.000 & 1.000
JFreechart (0.6.0) & 1.000 & 1.000
JFreechart (0.7.1) & 1.000 & 1.000
JHotDraw (7.4.1) & 0.999 & 0.999

Cohen's Kappa 3 WITH logs





In [56]:



    


print('-'*80)
print('\t\t CoffeeMaker')
print('-'*80)
j1 = Judge('leonardo.nole', 'CoffeeMaker')
j2 = Judge('rossella.linsalata', 'CoffeeMaker')
J = calculate_agreement_scores(j1, j2)
unweighted_k = cohens_kappa(J, log=True)
weighted_k = cohens_kappa(J, weighted=True, log=True)
print('Kappa: %.3f & %.3f' % (unweighted_k, weighted_k))

print('-'*80)
print('\t\t JfreeChart 0.6.0')
print('-'*80)
j1 = Judge('leonardo.nole', 'JFreechart', '0.6.0')
j2 = Judge('antonio.petrone', 'JFreechart', '0.6.0')
J = calculate_agreement_scores(j1, j2)
unweighted_k = cohens_kappa(J, log=True)
weighted_k = cohens_kappa(J, weighted=True, log=True)
print('Kappa: %.3f & %.3f' % (unweighted_k, weighted_k))

print('-'*80)
print('\t\t JfreeChart 0.7.1')
print('-'*80)
j1 = Judge('leonardo.nole', 'JFreechart', '0.7.1')
j2 = Judge('antonio.petrone', 'JFreechart', '0.7.1')
J = calculate_agreement_scores(j1, j2)
unweighted_k = cohens_kappa(J, log=True)
weighted_k = cohens_kappa(J, weighted=True, log=True)
print('Kappa: %.3f & %.3f' % (unweighted_k, weighted_k))

print('-'*80)
print('\t\t JHotDraw')
print('-'*80)
j1 = Judge('leonardo.nole', 'JHotDraw', '7.4.1')
j2 = Judge('rossella.linsalata', 'JHotDraw', '7.4.1')
J = calculate_agreement_scores(j1, j2)
unweighted_k = cohens_kappa(J, log=True)
weighted_k = cohens_kappa(J, weighted=True, log=True)
print('Kappa: %.3f & %.3f' % (unweighted_k, weighted_k))



    


















    






--------------------------------------------------------------------------------
		 CoffeeMaker
--------------------------------------------------------------------------------
J: 
 [[19  0  2]
 [ 0  0  0]
 [ 0  0 26]]
X: 
 [[ 0.40425532  0.          0.04255319]
 [ 0.          0.          0.        ]
 [ 0.          0.          0.55319149]]
W: 
 [[ 0.  1.  1.]
 [ 1.  0.  1.]
 [ 1.  1.  0.]]
AG: 
 [[ 21.   0.  26.]
 [ 19.   0.  28.]]
J_sum: 
 [[ 0.44680851  0.          0.55319149]
 [ 0.40425532  0.          0.59574468]]
M: 
 [[ 0.18062472  0.          0.26618379]
 [ 0.          0.          0.        ]
 [ 0.2236306   0.          0.32956089]]
J: 
 [[19  0  2]
 [ 0  0  0]
 [ 0  0 26]]
X: 
 [[ 0.40425532  0.          0.04255319]
 [ 0.          0.          0.        ]
 [ 0.          0.          0.55319149]]
W: 
 [[ 0.  1.  4.]
 [ 1.  0.  1.]
 [ 4.  1.  0.]]
AG: 
 [[ 21.   0.  26.]
 [ 19.   0.  28.]]
J_sum: 
 [[ 0.44680851  0.          0.55319149]
 [ 0.40425532  0.          0.59574468]]
M: 
 [[ 0.18062472  0.          0.26618379]
 [ 0.          0.          0.        ]
 [ 0.2236306   0.          0.32956089]]
Kappa: 0.913 & 0.913
--------------------------------------------------------------------------------
		 JfreeChart 0.6.0
--------------------------------------------------------------------------------
J: 
 [[ 47   0   0]
 [  0  24   0]
 [  8   0 406]]
X: 
 [[ 0.09690722  0.          0.        ]
 [ 0.          0.04948454  0.        ]
 [ 0.01649485  0.          0.8371134 ]]
W: 
 [[ 0.  1.  1.]
 [ 1.  0.  1.]
 [ 1.  1.  0.]]
AG: 
 [[  47.   24.  414.]
 [  55.   24.  406.]]
J_sum: 
 [[ 0.09690722  0.04948454  0.85360825]
 [ 0.11340206  0.04948454  0.8371134 ]]
M: 
 [[ 0.01098948  0.00479541  0.08112233]
 [ 0.00561165  0.00244872  0.04142417]
 [ 0.09680094  0.04224041  0.7145669 ]]
J: 
 [[ 47   0   0]
 [  0  24   0]
 [  8   0 406]]
X: 
 [[ 0.09690722  0.          0.        ]
 [ 0.          0.04948454  0.        ]
 [ 0.01649485  0.          0.8371134 ]]
W: 
 [[ 0.  1.  4.]
 [ 1.  0.  1.]
 [ 4.  1.  0.]]
AG: 
 [[  47.   24.  414.]
 [  55.   24.  406.]]
J_sum: 
 [[ 0.09690722  0.04948454  0.85360825]
 [ 0.11340206  0.04948454  0.8371134 ]]
M: 
 [[ 0.01098948  0.00479541  0.08112233]
 [ 0.00561165  0.00244872  0.04142417]
 [ 0.09680094  0.04224041  0.7145669 ]]
Kappa: 0.939 & 0.918
--------------------------------------------------------------------------------
		 JfreeChart 0.7.1
--------------------------------------------------------------------------------
J: 
 [[ 65   0   0]
 [  0  36   0]
 [  3   0 520]]
X: 
 [[ 0.10416667  0.          0.        ]
 [ 0.          0.05769231  0.        ]
 [ 0.00480769  0.          0.83333333]]
W: 
 [[ 0.  1.  1.]
 [ 1.  0.  1.]
 [ 1.  1.  0.]]
AG: 
 [[  65.   36.  523.]
 [  68.   36.  520.]]
J_sum: 
 [[ 0.10416667  0.05769231  0.83814103]
 [ 0.10897436  0.05769231  0.83333333]]
M: 
 [[ 0.0113515   0.00600962  0.08680556]
 [ 0.00628698  0.0033284   0.04807692]
 [ 0.09133588  0.04835429  0.69845085]]
J: 
 [[ 65   0   0]
 [  0  36   0]
 [  3   0 520]]
X: 
 [[ 0.10416667  0.          0.        ]
 [ 0.          0.05769231  0.        ]
 [ 0.00480769  0.          0.83333333]]
W: 
 [[ 0.  1.  4.]
 [ 1.  0.  1.]
 [ 4.  1.  0.]]
AG: 
 [[  65.   36.  523.]
 [  68.   36.  520.]]
J_sum: 
 [[ 0.10416667  0.05769231  0.83814103]
 [ 0.10897436  0.05769231  0.83333333]]
M: 
 [[ 0.0113515   0.00600962  0.08680556]
 [ 0.00628698  0.0033284   0.04807692]
 [ 0.09133588  0.04835429  0.69845085]]
Kappa: 0.983 & 0.977
--------------------------------------------------------------------------------
		 JHotDraw
--------------------------------------------------------------------------------
J: 
 [[808   0   0]
 [  0 676   0]
 [289   2 705]]
X: 
 [[ 0.32580645  0.          0.        ]
 [ 0.          0.27258065  0.        ]
 [ 0.11653226  0.00080645  0.28427419]]
W: 
 [[ 0.  1.  1.]
 [ 1.  0.  1.]
 [ 1.  1.  0.]]
AG: 
 [[  808.   676.   996.]
 [ 1097.   678.   705.]]
J_sum: 
 [[ 0.32580645  0.27258065  0.4016129 ]
 [ 0.44233871  0.2733871   0.28427419]]
M: 
 [[ 0.14411681  0.08907128  0.09261837]
 [ 0.12057297  0.07452003  0.07748764]
 [ 0.17764893  0.10979579  0.11416818]]
J: 
 [[808   0   0]
 [  0 676   0]
 [289   2 705]]
X: 
 [[ 0.32580645  0.          0.        ]
 [ 0.          0.27258065  0.        ]
 [ 0.11653226  0.00080645  0.28427419]]
W: 
 [[ 0.  1.  4.]
 [ 1.  0.  1.]
 [ 4.  1.  0.]]
AG: 
 [[  808.   676.   996.]
 [ 1097.   678.   705.]]
J_sum: 
 [[ 0.32580645  0.27258065  0.4016129 ]
 [ 0.44233871  0.2733871   0.28427419]]
M: 
 [[ 0.14411681  0.08907128  0.09261837]
 [ 0.12057297  0.07452003  0.07748764]
 [ 0.17764893  0.10979579  0.11416818]]
Kappa: 0.824 & 0.684

Cohen's Kappa 5 WITHOUT logs





In [57]:



    


j1 = Judge('leonardo.nole', 'CoffeeMaker')
j2 = Judge('rossella.linsalata', 'CoffeeMaker')
J = calculate_agreement_scores(j1, j2, k=5)
unweighted_k = cohens_kappa(J)
weighted_k = cohens_kappa(J, weighted=True)
print('CoffeeMaker & %.3f & %.3f' % (unweighted_k, weighted_k))

j1 = Judge('leonardo.nole', 'JFreechart', '0.6.0')
j2 = Judge('antonio.petrone', 'JFreechart', '0.6.0')
J = calculate_agreement_scores(j1, j2, k=5)
unweighted_k = cohens_kappa(J)
weighted_k = cohens_kappa(J, weighted=True)
print('JFreechart (0.6.0) & %.3f & %.3f' % (unweighted_k, weighted_k))

j1 = Judge('leonardo.nole', 'JFreechart', '0.7.1')
j2 = Judge('antonio.petrone', 'JFreechart', '0.7.1')
J = calculate_agreement_scores(j1, j2, k=5)
unweighted_k = cohens_kappa(J)
weighted_k = cohens_kappa(J, weighted=True)
print('JFreechart (0.7.1) & %.3f & %.3f' % (unweighted_k, weighted_k))

j1 = Judge('leonardo.nole', 'JHotDraw', '7.4.1')
j2 = Judge('rossella.linsalata', 'JHotDraw', '7.4.1')
J = calculate_agreement_scores(j1, j2, k=5)
unweighted_k = cohens_kappa(J)
weighted_k = cohens_kappa(J, weighted=True)
print('JHotDraw (7.4.1) & %.3f & %.3f' % (unweighted_k, weighted_k))



    


















    






CoffeeMaker & 0.282 & 0.807
JFreechart (0.6.0) & 0.202 & 0.657
JFreechart (0.7.1) & 0.163 & 0.669
JHotDraw (7.4.1) & 0.585 & 0.564

Cohen's Kappa 5 WITH logs





In [58]:



    


print('-'*80)
print('\t\t CoffeeMaker')
print('-'*80)
j1 = Judge('leonardo.nole', 'CoffeeMaker')
j2 = Judge('rossella.linsalata', 'CoffeeMaker')
J = calculate_agreement_scores(j1, j2, k=5)
unweighted_k = cohens_kappa(J, log=True)
weighted_k = cohens_kappa(J, weighted=True, log=True)
print('Kappa: %.3f & %.3f' % (unweighted_k, weighted_k))

print('-'*80)
print('\t\t JFreechart 0.6.0')
print('-'*80)
j1 = Judge('leonardo.nole', 'JFreechart', '0.6.0')
j2 = Judge('antonio.petrone', 'JFreechart', '0.6.0')
J = calculate_agreement_scores(j1, j2, k=5)
unweighted_k = cohens_kappa(J, log=True)
weighted_k = cohens_kappa(J, weighted=True, log=True)
print('Kappa: %.3f & %.3f' % (unweighted_k, weighted_k))

print('-'*80)
print('\t\t JFreechart 0.7.1')
print('-'*80)
j1 = Judge('leonardo.nole', 'JFreechart', '0.7.1')
j2 = Judge('antonio.petrone', 'JFreechart', '0.7.1')
J = calculate_agreement_scores(j1, j2, k=5)
unweighted_k = cohens_kappa(J, log=True)
weighted_k = cohens_kappa(J, weighted=True, log=True)
print('Kappa: %.3f & %.3f' % (unweighted_k, weighted_k))

print('-'*80)
print('\t\t JHotDraw')
print('-'*80)
j1 = Judge('leonardo.nole', 'JHotDraw', '7.4.1')
j2 = Judge('rossella.linsalata', 'JHotDraw', '7.4.1')
J = calculate_agreement_scores(j1, j2, k=5)
unweighted_k = cohens_kappa(J, log=True)
weighted_k = cohens_kappa(J, weighted=True, log=True)
print('Kappa: %.3f & %.3f' % (unweighted_k, weighted_k))



    


















    






--------------------------------------------------------------------------------
		 CoffeeMaker
--------------------------------------------------------------------------------
J: 
 [[ 1  5  0  1  0]
 [ 0 13  0  0  1]
 [ 0  0  0  0  0]
 [ 0  0  0  0  5]
 [ 0  0  0 12  9]]
X: 
 [[ 0.0212766   0.10638298  0.          0.0212766   0.        ]
 [ 0.          0.27659574  0.          0.          0.0212766 ]
 [ 0.          0.          0.          0.          0.        ]
 [ 0.          0.          0.          0.          0.10638298]
 [ 0.          0.          0.          0.25531915  0.19148936]]
W: 
 [[ 0.  1.  1.  1.  1.]
 [ 1.  0.  1.  1.  1.]
 [ 1.  1.  0.  1.  1.]
 [ 1.  1.  1.  0.  1.]
 [ 1.  1.  1.  1.  0.]]
AG: 
 [[  7.  14.   0.   5.  21.]
 [  1.  18.   0.  13.  15.]]
J_sum: 
 [[ 0.14893617  0.29787234  0.          0.10638298  0.44680851]
 [ 0.0212766   0.38297872  0.          0.27659574  0.31914894]]
M: 
 [[ 0.00316885  0.05703938  0.          0.04119511  0.04753282]
 [ 0.00633771  0.11407877  0.          0.08239022  0.09506564]
 [ 0.          0.          0.          0.          0.        ]
 [ 0.00226347  0.04074242  0.          0.02942508  0.03395201]
 [ 0.00950656  0.17111815  0.          0.12358533  0.14259846]]
J: 
 [[ 1  5  0  1  0]
 [ 0 13  0  0  1]
 [ 0  0  0  0  0]
 [ 0  0  0  0  5]
 [ 0  0  0 12  9]]
X: 
 [[ 0.0212766   0.10638298  0.          0.0212766   0.        ]
 [ 0.          0.27659574  0.          0.          0.0212766 ]
 [ 0.          0.          0.          0.          0.        ]
 [ 0.          0.          0.          0.          0.10638298]
 [ 0.          0.          0.          0.25531915  0.19148936]]
W: 
 [[  0.   1.   4.   9.  16.]
 [  1.   0.   1.   4.   9.]
 [  4.   1.   0.   1.   4.]
 [  9.   4.   1.   0.   1.]
 [ 16.   9.   4.   1.   0.]]
AG: 
 [[  7.  14.   0.   5.  21.]
 [  1.  18.   0.  13.  15.]]
J_sum: 
 [[ 0.14893617  0.29787234  0.          0.10638298  0.44680851]
 [ 0.0212766   0.38297872  0.          0.27659574  0.31914894]]
M: 
 [[ 0.00316885  0.05703938  0.          0.04119511  0.04753282]
 [ 0.00633771  0.11407877  0.          0.08239022  0.09506564]
 [ 0.          0.          0.          0.          0.        ]
 [ 0.00226347  0.04074242  0.          0.02942508  0.03395201]
 [ 0.00950656  0.17111815  0.          0.12358533  0.14259846]]
Kappa: 0.282 & 0.807
--------------------------------------------------------------------------------
		 JFreechart 0.6.0
--------------------------------------------------------------------------------
J: 
 [[  9  17   0   0   0]
 [  0  21   0   0   0]
 [  0   0  24   0   0]
 [  0   2   0  96   1]
 [  0   6   0 282  27]]
X: 
 [[ 0.0185567   0.03505155  0.          0.          0.        ]
 [ 0.          0.04329897  0.          0.          0.        ]
 [ 0.          0.          0.04948454  0.          0.        ]
 [ 0.          0.00412371  0.          0.19793814  0.00206186]
 [ 0.          0.01237113  0.          0.5814433   0.0556701 ]]
W: 
 [[ 0.  1.  1.  1.  1.]
 [ 1.  0.  1.  1.  1.]
 [ 1.  1.  0.  1.  1.]
 [ 1.  1.  1.  0.  1.]
 [ 1.  1.  1.  1.  0.]]
AG: 
 [[  26.   21.   24.   99.  315.]
 [   9.   46.   24.  378.   28.]]
J_sum: 
 [[ 0.05360825  0.04329897  0.04948454  0.20412371  0.64948454]
 [ 0.0185567   0.09484536  0.04948454  0.77938144  0.05773196]]
M: 
 [[ 0.00099479  0.00508449  0.00265278  0.04178127  0.00309491]
 [ 0.00080349  0.00410671  0.00214263  0.03374641  0.00249973]
 [ 0.00091827  0.00469338  0.00244872  0.03856733  0.00285684]
 [ 0.00378786  0.01936019  0.01010097  0.15909023  0.01178446]
 [ 0.01205229  0.0616006   0.03213944  0.5061962   0.03749601]]
J: 
 [[  9  17   0   0   0]
 [  0  21   0   0   0]
 [  0   0  24   0   0]
 [  0   2   0  96   1]
 [  0   6   0 282  27]]
X: 
 [[ 0.0185567   0.03505155  0.          0.          0.        ]
 [ 0.          0.04329897  0.          0.          0.        ]
 [ 0.          0.          0.04948454  0.          0.        ]
 [ 0.          0.00412371  0.          0.19793814  0.00206186]
 [ 0.          0.01237113  0.          0.5814433   0.0556701 ]]
W: 
 [[  0.   1.   4.   9.  16.]
 [  1.   0.   1.   4.   9.]
 [  4.   1.   0.   1.   4.]
 [  9.   4.   1.   0.   1.]
 [ 16.   9.   4.   1.   0.]]
AG: 
 [[  26.   21.   24.   99.  315.]
 [   9.   46.   24.  378.   28.]]
J_sum: 
 [[ 0.05360825  0.04329897  0.04948454  0.20412371  0.64948454]
 [ 0.0185567   0.09484536  0.04948454  0.77938144  0.05773196]]
M: 
 [[ 0.00099479  0.00508449  0.00265278  0.04178127  0.00309491]
 [ 0.00080349  0.00410671  0.00214263  0.03374641  0.00249973]
 [ 0.00091827  0.00469338  0.00244872  0.03856733  0.00285684]
 [ 0.00378786  0.01936019  0.01010097  0.15909023  0.01178446]
 [ 0.01205229  0.0616006   0.03213944  0.5061962   0.03749601]]
Kappa: 0.202 & 0.657
--------------------------------------------------------------------------------
		 JFreechart 0.7.1
--------------------------------------------------------------------------------
J: 
 [[ 11  15   0   0   0]
 [  4  35   0   0   0]
 [  0   0  36   0   0]
 [  0   3   0  65   0]
 [  0   0   0 440  15]]
X: 
 [[ 0.01762821  0.02403846  0.          0.          0.        ]
 [ 0.00641026  0.05608974  0.          0.          0.        ]
 [ 0.          0.          0.05769231  0.          0.        ]
 [ 0.          0.00480769  0.          0.10416667  0.        ]
 [ 0.          0.          0.          0.70512821  0.02403846]]
W: 
 [[ 0.  1.  1.  1.  1.]
 [ 1.  0.  1.  1.  1.]
 [ 1.  1.  0.  1.  1.]
 [ 1.  1.  1.  0.  1.]
 [ 1.  1.  1.  1.  0.]]
AG: 
 [[  26.   39.   36.   68.  455.]
 [  15.   53.   36.  505.   15.]]
J_sum: 
 [[ 0.04166667  0.0625      0.05769231  0.10897436  0.72916667]
 [ 0.02403846  0.0849359   0.05769231  0.80929487  0.02403846]]
M: 
 [[ 0.0010016   0.003539    0.00240385  0.03372062  0.0010016 ]
 [ 0.0015024   0.00530849  0.00360577  0.05058093  0.0015024 ]
 [ 0.00138683  0.00490015  0.0033284   0.04669009  0.00138683]
 [ 0.00261958  0.00925583  0.00628698  0.08819239  0.00261958]
 [ 0.01752804  0.06193243  0.04206731  0.59011084  0.01752804]]
J: 
 [[ 11  15   0   0   0]
 [  4  35   0   0   0]
 [  0   0  36   0   0]
 [  0   3   0  65   0]
 [  0   0   0 440  15]]
X: 
 [[ 0.01762821  0.02403846  0.          0.          0.        ]
 [ 0.00641026  0.05608974  0.          0.          0.        ]
 [ 0.          0.          0.05769231  0.          0.        ]
 [ 0.          0.00480769  0.          0.10416667  0.        ]
 [ 0.          0.          0.          0.70512821  0.02403846]]
W: 
 [[  0.   1.   4.   9.  16.]
 [  1.   0.   1.   4.   9.]
 [  4.   1.   0.   1.   4.]
 [  9.   4.   1.   0.   1.]
 [ 16.   9.   4.   1.   0.]]
AG: 
 [[  26.   39.   36.   68.  455.]
 [  15.   53.   36.  505.   15.]]
J_sum: 
 [[ 0.04166667  0.0625      0.05769231  0.10897436  0.72916667]
 [ 0.02403846  0.0849359   0.05769231  0.80929487  0.02403846]]
M: 
 [[ 0.0010016   0.003539    0.00240385  0.03372062  0.0010016 ]
 [ 0.0015024   0.00530849  0.00360577  0.05058093  0.0015024 ]
 [ 0.00138683  0.00490015  0.0033284   0.04669009  0.00138683]
 [ 0.00261958  0.00925583  0.00628698  0.08819239  0.00261958]
 [ 0.01752804  0.06193243  0.04206731  0.59011084  0.01752804]]
Kappa: 0.163 & 0.669
--------------------------------------------------------------------------------
		 JHotDraw
--------------------------------------------------------------------------------
J: 
 [[ 70  45   0   0   0]
 [ 15 678   0   0   0]
 [  0   0 676   0   0]
 [  0  35   1 235   1]
 [  0 254   1 448  21]]
X: 
 [[ 0.02822581  0.01814516  0.          0.          0.        ]
 [ 0.00604839  0.2733871   0.          0.          0.        ]
 [ 0.          0.          0.27258065  0.          0.        ]
 [ 0.          0.0141129   0.00040323  0.09475806  0.00040323]
 [ 0.          0.10241935  0.00040323  0.18064516  0.00846774]]
W: 
 [[ 0.  1.  1.  1.  1.]
 [ 1.  0.  1.  1.  1.]
 [ 1.  1.  0.  1.  1.]
 [ 1.  1.  1.  0.  1.]
 [ 1.  1.  1.  1.  0.]]
AG: 
 [[  115.   693.   676.   272.   724.]
 [   85.  1012.   678.   683.    22.]]
J_sum: 
 [[ 0.04637097  0.27943548  0.27258065  0.10967742  0.29193548]
 [ 0.03427419  0.40806452  0.2733871   0.27540323  0.00887097]]
M: 
 [[ 0.00158933  0.01892235  0.01267722  0.01277071  0.00041136]
 [ 0.00957743  0.11402771  0.07639406  0.07695743  0.00247886]
 [ 0.00934248  0.11123049  0.07452003  0.07506959  0.00241805]
 [ 0.00375911  0.04475546  0.02998439  0.03020552  0.00097294]
 [ 0.01000585  0.11912851  0.07981139  0.08039997  0.00258975]]
J: 
 [[ 70  45   0   0   0]
 [ 15 678   0   0   0]
 [  0   0 676   0   0]
 [  0  35   1 235   1]
 [  0 254   1 448  21]]
X: 
 [[ 0.02822581  0.01814516  0.          0.          0.        ]
 [ 0.00604839  0.2733871   0.          0.          0.        ]
 [ 0.          0.          0.27258065  0.          0.        ]
 [ 0.          0.0141129   0.00040323  0.09475806  0.00040323]
 [ 0.          0.10241935  0.00040323  0.18064516  0.00846774]]
W: 
 [[  0.   1.   4.   9.  16.]
 [  1.   0.   1.   4.   9.]
 [  4.   1.   0.   1.   4.]
 [  9.   4.   1.   0.   1.]
 [ 16.   9.   4.   1.   0.]]
AG: 
 [[  115.   693.   676.   272.   724.]
 [   85.  1012.   678.   683.    22.]]
J_sum: 
 [[ 0.04637097  0.27943548  0.27258065  0.10967742  0.29193548]
 [ 0.03427419  0.40806452  0.2733871   0.27540323  0.00887097]]
M: 
 [[ 0.00158933  0.01892235  0.01267722  0.01277071  0.00041136]
 [ 0.00957743  0.11402771  0.07639406  0.07695743  0.00247886]
 [ 0.00934248  0.11123049  0.07452003  0.07506959  0.00241805]
 [ 0.00375911  0.04475546  0.02998439  0.03020552  0.00097294]
 [ 0.01000585  0.11912851  0.07981139  0.08039997  0.00258975]]
Kappa: 0.585 & 0.564

Cohens' Kappa 2 WITHOUT logs





In [59]:



    


# ------
# NOTE: In this case Weighted and Unweighted are exactly the same
# ------

j1 = Judge('leonardo.nole', 'CoffeeMaker')
j2 = Judge('rossella.linsalata', 'CoffeeMaker')
J = calculate_agreement_scores(j1, j2, k=2)
unweighted_k = cohens_kappa(J)
# weighted_k = cohens_kappa(J, weighted=True)
print('CoffeeMaker & %.3f' % (unweighted_k))

j1 = Judge('leonardo.nole', 'JFreechart', '0.6.0')
j2 = Judge('antonio.petrone', 'JFreechart', '0.6.0')
J = calculate_agreement_scores(j1, j2, k=2)
unweighted_k = cohens_kappa(J)
# weighted_k = cohens_kappa(J, weighted=True)
print('JFreechart (0.6.0) & %.3f' % (unweighted_k))

j1 = Judge('leonardo.nole', 'JFreechart', '0.7.1')
j2 = Judge('antonio.petrone', 'JFreechart', '0.7.1')
J = calculate_agreement_scores(j1, j2, k=2)
unweighted_k = cohens_kappa(J)
# weighted_k = cohens_kappa(J, weighted=True)
print('JFreechart (0.7.1) & %.3f' % (unweighted_k,))

j1 = Judge('leonardo.nole', 'JHotDraw', '7.4.1')
j2 = Judge('rossella.linsalata', 'JHotDraw', '7.4.1')
J = calculate_agreement_scores(j1, j2, k=2)
unweighted_k = cohens_kappa(J)
# weighted_k = cohens_kappa(J, weighted=True)
print('JHotDraw (7.4.1) & %.3f' % (unweighted_k))



    


















    






CoffeeMaker & 0.913
JFreechart (0.6.0) & 0.912
JFreechart (0.7.1) & 0.975
JHotDraw (7.4.1) & 0.686

Cohen's Kappa 2 WITH logs





In [60]:



    


# ------
# NOTE: In this case Weighted and Unweighted are exactly the same
# ------

j1 = Judge('leonardo.nole', 'CoffeeMaker')
j2 = Judge('rossella.linsalata', 'CoffeeMaker')
J = calculate_agreement_scores(j1, j2, k=2)
unweighted_k = cohens_kappa(J, log=True)
# weighted_k = cohens_kappa(J, weighted=True)
print('CoffeeMaker & %.3f' % (unweighted_k))

j1 = Judge('leonardo.nole', 'JFreechart', '0.6.0')
j2 = Judge('antonio.petrone', 'JFreechart', '0.6.0')
J = calculate_agreement_scores(j1, j2, k=2)
unweighted_k = cohens_kappa(J, log=True)
# weighted_k = cohens_kappa(J, weighted=True)
print('JFreechart (0.6.0) & %.3f' % (unweighted_k))

j1 = Judge('leonardo.nole', 'JFreechart', '0.7.1')
j2 = Judge('antonio.petrone', 'JFreechart', '0.7.1')
J = calculate_agreement_scores(j1, j2, k=2)
unweighted_k = cohens_kappa(J, log=True)
# weighted_k = cohens_kappa(J, weighted=True)
print('JFreechart (0.7.1) & %.3f' % (unweighted_k,))

j1 = Judge('leonardo.nole', 'JHotDraw', '7.4.1')
j2 = Judge('rossella.linsalata', 'JHotDraw', '7.4.1')
J = calculate_agreement_scores(j1, j2, k=2)
unweighted_k = cohens_kappa(J, log=True)
# weighted_k = cohens_kappa(J, weighted=True)
print('JHotDraw (7.4.1) & %.3f' % (unweighted_k))



    


















    






J: 
 [[19  2]
 [ 0 26]]
X: 
 [[ 0.40425532  0.04255319]
 [ 0.          0.55319149]]
W: 
 [[ 0.  1.]
 [ 1.  0.]]
AG: 
 [[ 21.  26.]
 [ 19.  28.]]
J_sum: 
 [[ 0.44680851  0.55319149]
 [ 0.40425532  0.59574468]]
M: 
 [[ 0.18062472  0.26618379]
 [ 0.2236306   0.32956089]]
CoffeeMaker & 0.913
J: 
 [[ 47   0]
 [  8 406]]
X: 
 [[ 0.10195228  0.        ]
 [ 0.01735358  0.88069414]]
W: 
 [[ 0.  1.]
 [ 1.  0.]]
AG: 
 [[  47.  414.]
 [  55.  406.]]
J_sum: 
 [[ 0.10195228  0.89804772]
 [ 0.11930586  0.88069414]]
M: 
 [[ 0.0121635   0.08978877]
 [ 0.10714235  0.79090537]]
JFreechart (0.6.0) & 0.912
J: 
 [[ 65   0]
 [  3 520]]
X: 
 [[ 0.11054422  0.        ]
 [ 0.00510204  0.88435374]]
W: 
 [[ 0.  1.]
 [ 1.  0.]]
AG: 
 [[  65.  523.]
 [  68.  520.]]
J_sum: 
 [[ 0.11054422  0.88945578]
 [ 0.11564626  0.88435374]]
M: 
 [[ 0.01278403  0.09776019]
 [ 0.10286223  0.78659355]]
JFreechart (0.7.1) & 0.975
J: 
 [[808   0]
 [289 705]]
X: 
 [[ 0.44839068  0.        ]
 [ 0.16037736  0.39123196]]
W: 
 [[ 0.  1.]
 [ 1.  0.]]
AG: 
 [[  808.   994.]
 [ 1097.   705.]]
J_sum: 
 [[ 0.44839068  0.55160932]
 [ 0.60876804  0.39123196]]
M: 
 [[ 0.27296591  0.17542477]
 [ 0.33580212  0.2158072 ]]
JHotDraw (7.4.1) & 0.686

Calculate the Mean Precision of Judges' evaluations





In [8]:



    


from evaluations import mean_precision

Mean Precision "Coherent"





In [61]:



    


j1 = Judge('leonardo.nole', 'CoffeeMaker')
j2 = Judge('rossella.linsalata', 'CoffeeMaker')
j1_eval = j1.three_codes_evaluations[2]
j2_eval = j2.three_codes_evaluations[2]
pj1, pj2, f = mean_precision(j1_eval, j2_eval)
print('CoffeeMaker & %.3f & %.3f & %.3f' % (pj1, pj2, f))

j1 = Judge('leonardo.nole', 'JFreechart', '0.6.0')
j2 = Judge('antonio.petrone', 'JFreechart', '0.6.0')
j1_eval = j1.three_codes_evaluations[2]
j2_eval = j2.three_codes_evaluations[2]
pj1, pj2, f = mean_precision(j1_eval, j2_eval)
print('JFreeChart (0.6.0) & %.3f & %.3f & %.3f' % (pj1, pj2, f))

j1 = Judge('leonardo.nole', 'JFreechart', '0.7.1')
j2 = Judge('antonio.petrone', 'JFreechart', '0.7.1')
j1_eval = j1.three_codes_evaluations[2]
j2_eval = j2.three_codes_evaluations[2]
pj1, pj2, f = mean_precision(j1_eval, j2_eval)
print('JFreeChart (0.7.1) & %.3f & %.3f & %.3f' % (pj1, pj2, f))

j1 = Judge('leonardo.nole', 'JHotDraw', '7.4.1')
j2 = Judge('rossella.linsalata', 'JHotDraw', '7.4.1')
j1_eval = j1.three_codes_evaluations[2]
j2_eval = j2.three_codes_evaluations[2]
pj1, pj2, f = mean_precision(j1_eval, j2_eval)
print('JHotDraw (7.4.1) & %.3f & %.3f & %.3f' % (pj1, pj2, f))



    


















    






CoffeeMaker & 1.000 & 0.929 & 0.963
JFreeChart (0.6.0) & 0.981 & 1.000 & 0.990
JFreeChart (0.7.1) & 0.994 & 1.000 & 0.997
JHotDraw (7.4.1) & 0.708 & 1.000 & 0.829

Mean Precision "Non Coherent"





In [62]:



    


j1 = Judge('leonardo.nole', 'CoffeeMaker')
j2 = Judge('rossella.linsalata', 'CoffeeMaker')
j1_eval = j1.three_codes_evaluations[0]
j2_eval = j2.three_codes_evaluations[0]
pj1, pj2, f = mean_precision(j1_eval, j2_eval)
print('CoffeeMaker & %.3f & %.3f & %.3f' % (pj1, pj2, f))

j1 = Judge('leonardo.nole', 'JFreechart', '0.6.0')
j2 = Judge('antonio.petrone', 'JFreechart', '0.6.0')
j1_eval = j1.three_codes_evaluations[0]
j2_eval = j2.three_codes_evaluations[0]
pj1, pj2, f = mean_precision(j1_eval, j2_eval)
print('JFreeChart (0.6.0) & %.3f & %.3f & %.3f' % (pj1, pj2, f))

j1 = Judge('leonardo.nole', 'JFreechart', '0.7.1')
j2 = Judge('antonio.petrone', 'JFreechart', '0.7.1')
j1_eval = j1.three_codes_evaluations[0]
j2_eval = j2.three_codes_evaluations[0]
pj1, pj2, f = mean_precision(j1_eval, j2_eval)
print('JFreeChart (0.7.1) & %.3f & %.3f & %.3f' % (pj1, pj2, f))

j1 = Judge('leonardo.nole', 'JHotDraw', '7.4.1')
j2 = Judge('rossella.linsalata', 'JHotDraw', '7.4.1')
j1_eval = j1.three_codes_evaluations[0]
j2_eval = j2.three_codes_evaluations[0]
pj1, pj2, f = mean_precision(j1_eval, j2_eval)
print('JHotDraw (7.4.1) & %.3f & %.3f & %.3f' % (pj1, pj2, f))



    


















    






CoffeeMaker & 0.905 & 1.000 & 0.950
JFreeChart (0.6.0) & 1.000 & 0.855 & 0.922
JFreeChart (0.7.1) & 1.000 & 0.956 & 0.977
JHotDraw (7.4.1) & 1.000 & 0.737 & 0.848

Check the Differences





In [5]:



    


from source_code_analysis.models import AgreementEvaluation, SoftwareProject
from django.contrib.auth.models import User

CoffeeMaker





In [6]:



    


j1 = Judge('leonardo.nole', 'CoffeeMaker')
j2 = Judge('rossella.linsalata', 'CoffeeMaker')

j1_evals = j1.two_codes_evaluations
j2_evals = j2.two_codes_evaluations

neg_diff = j1_evals[0].intersection(j2_evals[1])
pos_diff = j1_evals[1].intersection(j2_evals[0])

leo = User.objects.get(username='leonardo.nole')
ros = User.objects.get(username='rossella.linsalata')

# -------------------------
# NEG
# -------------------------

neg_id_list = list()
for meth_id in neg_diff:
    ag_eval = AgreementEvaluation.objects.get(reference_method__id=meth_id, evaluator=leo)
    neg_id_list.append(str(ag_eval.pk))
print('J1:', ','.join(neg_id_list))

neg_id_list = list()
for meth_id in neg_diff:
    ag_eval = AgreementEvaluation.objects.get(reference_method__id=meth_id, evaluator=ros)
    neg_id_list.append(str(ag_eval.pk))
print('J2:', ','.join(neg_id_list))

# -------------------------
# POS
# -------------------------
pos_id_list = list()
for meth_id in pos_diff:
    ag_eval = AgreementEvaluation.objects.get(reference_method__id=meth_id, evaluator=leo)
    pos_id_list.append(str(ag_eval.pk))
print('J1:', ','.join(pos_id_list))

pos_id_list = list()
for meth_id in pos_diff:
    ag_eval = AgreementEvaluation.objects.get(reference_method__id=meth_id, evaluator=ros)
    pos_id_list.append(str(ag_eval.pk))
print('J2:', ','.join(pos_id_list))



    


















    






J1: 
J2: 
J1: 
J2:

JFreeChart 0.6.0





In [7]:



    


j1 = Judge('leonardo.nole', 'JFreechart', '0.6.0')
j2 = Judge('antonio.petrone', 'JFreechart', '0.6.0')

j1_evals = j1.two_codes_evaluations
j2_evals = j2.two_codes_evaluations

neg_diff = j1_evals[0].intersection(j2_evals[1])
pos_diff = j1_evals[1].intersection(j2_evals[0])

leo = User.objects.get(username='leonardo.nole')
anto = User.objects.get(username='antonio.petrone')

# -------------------------
# NEG
# -------------------------

neg_id_list = list()
for meth_id in neg_diff:
    ag_eval = AgreementEvaluation.objects.get(reference_method__id=meth_id, evaluator=leo)
    neg_id_list.append(str(ag_eval.pk))
print('J1:', ','.join(neg_id_list))

neg_id_list = list()
for meth_id in neg_diff:
    ag_eval = AgreementEvaluation.objects.get(reference_method__id=meth_id, evaluator=anto)
    neg_id_list.append(str(ag_eval.pk))
print('J2:', ','.join(neg_id_list))

# -------------------------
# POS
# -------------------------
pos_id_list = list()
for meth_id in pos_diff:
    ag_eval = AgreementEvaluation.objects.get(reference_method__id=meth_id, evaluator=leo)
    pos_id_list.append(str(ag_eval.pk))
print('J1:', ','.join(pos_id_list))

pos_id_list = list()
for meth_id in pos_diff:
    ag_eval = AgreementEvaluation.objects.get(reference_method__id=meth_id, evaluator=anto)
    pos_id_list.append(str(ag_eval.pk))
print('J2:', ','.join(pos_id_list))



    


















    






J1: 
J2: 
J1: 
J2:
JFreeChart 0.7.1




In [8]:



    


j1 = Judge('leonardo.nole', 'JFreechart', '0.7.1')
j2 = Judge('antonio.petrone', 'JFreechart', '0.7.1')

j1_evals = j1.two_codes_evaluations
j2_evals = j2.two_codes_evaluations

neg_diff = j1_evals[0].intersection(j2_evals[1])
pos_diff = j1_evals[1].intersection(j2_evals[0])

leo = User.objects.get(username='leonardo.nole')
anto = User.objects.get(username='antonio.petrone')

# -------------------------
# NEG
# -------------------------

neg_id_list = list()
for meth_id in neg_diff:
    ag_eval = AgreementEvaluation.objects.get(reference_method__id=meth_id, evaluator=leo)
    neg_id_list.append(str(ag_eval.pk))
print('J1:', ','.join(neg_id_list))

neg_id_list = list()
for meth_id in neg_diff:
    ag_eval = AgreementEvaluation.objects.get(reference_method__id=meth_id, evaluator=anto)
    neg_id_list.append(str(ag_eval.pk))
print('J2:', ','.join(neg_id_list))

# -------------------------
# POS
# -------------------------
pos_id_list = list()
for meth_id in pos_diff:
    ag_eval = AgreementEvaluation.objects.get(reference_method__id=meth_id, evaluator=leo)
    pos_id_list.append(str(ag_eval.pk))
print('J1:', ','.join(pos_id_list))

pos_id_list = list()
for meth_id in pos_diff:
    ag_eval = AgreementEvaluation.objects.get(reference_method__id=meth_id, evaluator=anto)
    pos_id_list.append(str(ag_eval.pk))
print('J2:', ','.join(pos_id_list))



    


















    






J1: 
J2: 
J1: 
J2:

JHotDraw 7.4.1





In [10]:



    


j1 = Judge('leonardo.nole', 'JHotDraw', '7.4.1')
j2 = Judge('rossella.linsalata', 'JHotDraw', '7.4.1')

j1_evals = j1.two_codes_evaluations
j2_evals = j2.two_codes_evaluations

neg_diff = j1_evals[0].intersection(j2_evals[1])
pos_diff = j1_evals[1].intersection(j2_evals[0])

leo = User.objects.get(username='leonardo.nole')
anto = User.objects.get(username='rossella.linsalata')

# -------------------------
# NEG
# -------------------------

neg_id_list = list()
for meth_id in neg_diff:
    ag_eval = AgreementEvaluation.objects.get(reference_method__id=meth_id, evaluator=leo)
    neg_id_list.append(str(ag_eval.pk))
print('J1:', ','.join(neg_id_list))

neg_id_list = list()
for meth_id in neg_diff:
    ag_eval = AgreementEvaluation.objects.get(reference_method__id=meth_id, evaluator=anto)
    neg_id_list.append(str(ag_eval.pk))
print('J2:', ','.join(neg_id_list))

# -------------------------
# POS
# -------------------------
pos_id_list = list()
for meth_id in pos_diff:
    ag_eval = AgreementEvaluation.objects.get(reference_method__id=meth_id, evaluator=leo)
    pos_id_list.append(str(ag_eval.pk))
print('J1:', ','.join(pos_id_list))

pos_id_list = list()
for meth_id in pos_diff:
    ag_eval = AgreementEvaluation.objects.get(reference_method__id=meth_id, evaluator=anto)
    pos_id_list.append(str(ag_eval.pk))
    
print('J2:', ','.join(pos_id_list))



    


















    






J1: 
J2: 
J1: 
J2:

TEST: Lexical overlap considering ONLY the Intersection agreement





In [33]:



    


from sklearn.feature_extraction.text import TfidfVectorizer


judges_combinations = (('leonardo.nole', 'rossella.linsalata'),
                       ('leonardo.nole', 'rossella.linsalata'),
                       ('leonardo.nole', 'antonio.petrone'),
                       ('leonardo.nole', 'antonio.petrone'),)

CODES_Labels = ('NC', 'DK', 'CO')
from collections import defaultdict
stats_results = defaultdict(list)

for pno, project in enumerate(projects):
    
    if not pno == 1:
        continue

    # Get Methods
    code_methods = project.code_methods.all()

    # Populate the Doc Collection
    document_collection = list()
    method_ids_map = dict()  # Map (dict) to store the association method.pk --> Row index in Tfidf Matrix
    for mno, method in enumerate(code_methods):
        clexicon_info = method.lexical_info
        document_collection.append(clexicon_info.normalized_comment)
        document_collection.append(clexicon_info.normalized_code)
        method_ids_map[method.id] = mno * 2

    vectorizer = TfidfVectorizer(input='content', sublinear_tf=True, lowercase=False)
    tfidf_values = vectorizer.fit_transform(document_collection)

    j1_usrname, j2_usrname = judges_combinations[pno]
    j1 = Judge(j1_usrname, project.name, project.version)
    j2 = Judge(j2_usrname, project.name, project.version)
    
    j1_evals = j1.three_codes_evaluations
    j2_evals = j2.three_codes_evaluations
    
    project_stats = list()
    method_ids = list()
    for code in range(3):
        j1_evals_code = j1_evals[code]
        j2_evals_code = j2_evals[code]
        
        method_ids.extend(j1_evals_code.intersection(j2_evals_code))
        
    cosine_sim_vals = list()
    for mid in method_ids:
        i = method_ids_map[mid]
        assert i % 2 == 0, print(i, mid)
        dotprod = tfidf_values[i].dot(tfidf_values[i+1].T)[0,0]
        cosine_sim_vals.append(dotprod)
        if dotprod == 1.0:
            print('MID: ', mid)
    
    vals = np.array(cosine_sim_vals)
    print('{proj} ({ver}) & {total} & {min:.3} & {max:.3} & {median:.3} & {mean:.3} & {variance:.3} & {devstd:.3} \\\\'.format(
                                                                                 proj = project.name.title(), 
                                                                                 ver=project.version,
                                                                                 total=vals.size, 
                                                                                 min=vals.min(), 
                                                                                 max=vals.max(), 
                                                                                 median=median(vals), 
                                                                                 mean=vals.mean(), 
                                                                                 variance=var(vals), 
                                                                                 devstd=std(vals)))



    


















    






MID:  980
Jhotdraw (7.4.1) & 2189 & 0.0 & 1.0 & 0.408 & 0.397 & 0.0635 & 0.252 \\

TEST: Lexical overlap considering ONLY the Methods were Judges Did not Agree on their Coherence Value





In [30]:



    


from sklearn.feature_extraction.text import TfidfVectorizer


judges_combinations = (('leonardo.nole', 'rossella.linsalata'),
                       ('leonardo.nole', 'rossella.linsalata'),
                       ('leonardo.nole', 'antonio.petrone'),
                       ('leonardo.nole', 'antonio.petrone'),)

CODES_Labels = ('NC', 'DK', 'CO')
from collections import defaultdict
stats_results = defaultdict(list)

for pno, project in enumerate(projects):

    # Get Methods
    code_methods = project.code_methods.all()

    # Populate the Doc Collection
    document_collection = list()
    method_ids_map = dict()  # Map (dict) to store the association method.pk --> Row index in Tfidf Matrix
    for mno, method in enumerate(code_methods):
        clexicon_info = method.lexical_info
        document_collection.append(clexicon_info.normalized_comment)
        document_collection.append(clexicon_info.normalized_code)
        method_ids_map[method.id] = mno * 2

    vectorizer = TfidfVectorizer(input='content', sublinear_tf=True, lowercase=False)
    tfidf_values = vectorizer.fit_transform(document_collection)

    j1_usrname, j2_usrname = judges_combinations[pno]
    j1 = Judge(j1_usrname, project.name, project.version)
    j2 = Judge(j2_usrname, project.name, project.version)
    
    j1_evals = j1.three_codes_evaluations
    j2_evals = j2.three_codes_evaluations
    
    project_stats = list()
    method_ids = list()
    for code in range(3):
        j1_evals_code = j1_evals[code]
        j2_evals_code = j2_evals[code]
        
        method_ids.extend(j1_evals_code.intersection(j2_evals_code))
        
    cosine_sim_vals = list()
    for mid in method_ids_map:
        if not mid in method_ids:
            i = method_ids_map[mid]
            cosine_sim_vals.append(tfidf_values[i].dot(tfidf_values[i+1].T)[0,0])
    
    vals = np.array(cosine_sim_vals)
    print('{proj} ({ver}) & {total} & {min:.3} & {max:.3} & {median:.3} & {mean:.3} & {variance:.3} & {devstd:.3} \\\\'.format(
                                                                                 proj = project.name.title(), 
                                                                                 ver=project.version,
                                                                                 total=vals.size, 
                                                                                 min=vals.min(), 
                                                                                 max=vals.max(), 
                                                                                 median=median(vals), 
                                                                                 mean=vals.mean(), 
                                                                                 variance=var(vals), 
                                                                                 devstd=std(vals)))



    


















    






Coffeemaker (1.0) & 2 & 0.628 & 0.703 & 0.665 & 0.665 & 0.0014 & 0.0375 \\
Jhotdraw (7.4.1) & 504 & 0.0 & 0.927 & 0.396 & 0.397 & 0.0612 & 0.247 \\
Jfreechart (0.6.0) & 14 & 0.0 & 0.582 & 0.266 & 0.251 & 0.0553 & 0.235 \\
Jfreechart (0.7.1) & 11 & 0.0 & 0.258 & 0.0 & 0.0792 & 0.0112 & 0.106 \\

Content source: leriomaggio/code-coherence-analysis

Similar notebooks:

notebook.community | gallery | about